<?php
/* This class accepts an HTML address and can return a string array containing the text.
 * It removes HTML tags and Javascripts using a built-in PHP function and regex.
 * Generally, this is a VERY slow process, considering the amount of test data.
 * This class also calls the SQClassifier class, which classifies the webpage given.
 * This class can also return the classification of the given webpage.
 * 
 * Categories: Leisure, Academics
 *
 * Author: Baron Gerarld S. Manzano
 * 
 */

include "sqclassifier.php";

class SQCrawler {

	protected $markup = '';
	protected $content = '';
	protected $words = array();
	protected $classtype = '';
	protected $url = '';
	
	protected $deleteme = 0;
	
	/*
	 * This function instantiates the SQCrawler class given the URL of the webpage to be crawled.
	 * It crawls the webpage and gathers the words in an array, arranges it, and gives it to an instantiated SQClassifier class.
	 * Afterwards, it sets the classification type of the webpage crawled.
	 */
	public function __construct($uri, $num) {
		set_time_limit(30);
		$this->content = '';
		$this->classtype = 'unclassifiable';
		
		
		$record = $this->checkRecorded($num);
		if($record != NULL){
			$this->classtype = $record;
			return;
		}
		else{
		}

		try{
		@$this->markup = $this->getContent($uri);
		}
		catch(Exception $e){
			echo "No data received.";
			$this->markup = false;
		}
		if($this->markup === false){
			return;
		}
		if(empty($this->markup)){
			return;
		}
		
		
		$this->content = strip_tags($this->markup, '<script>');									// for removing HTML
		$this->content = preg_replace('#(<script.*?>).*?(</script>)#s', '', $this->content);	// and Javascript
		$this->content = preg_replace('/[^\w\d_ -]/si', ' ', $this->content);					// for removing special characters
		$this->content = preg_replace('/-/', ' ', $this->content);
		
		$this->getTextArray();
		
		$classifier = new SQClassifier($this->words);
		$this->classtype = $classifier->getClassification();
		
		$con = mysql_connect("localhost","sqadmin","squid");
		if (!$con){
			die("Could not connect: " . mysql_error());
		}
		else{
			mysql_select_db("squidler", $con);
			$info = mysql_query("UPDATE logdata SET webpageclass = '$this->classtype' WHERE webpageid='$num'");
		}
		mysql_close($con);
		
		unset($classifier);
	}

	/*
	 * This function crawls the website and gets the HTML source file.
	 */
	private function getContent($uri) {
		$context = stream_context_create(array('http' => array('timeout' => 10, 'header'=>'Connection: close')));
		
		return @file_get_contents($uri, 0, $context);
	}
	
	/*
	 * This function gets the array of words of at least 5 characters of length.
	 */
	public function getTextArray() {
		$temp = strtok($this->content, ' ');
		
		while($temp != false){
			if(strlen($temp) > 4){
				$this->words[] = $temp;
			}
			
			$temp = strtok(' ');
		}
		
		return $this->words;
	}
	
	/*
	 * This function checks the database if the webpage index has already been classified.
	 */
	private function checkRecorded($num){
		$con = mysql_connect("localhost","sqadmin","squid");
		if (!$con){
			die("Could not connect: " . mysql_error());
			return;
		}
		mysql_select_db("squidler", $con);
		$info = mysql_query("SELECT webpageclass FROM logdata WHERE webpageid='$num'");
		$result = mysql_num_rows($info);
		mysql_close($con);
		if($result > 0){
			$result = mysql_fetch_row($info);
			return $result[0];
		}
		return NULL;
	}
	
	/*
	 * Getter function for the webpage's classification.
	 */
	public function getClass(){
		return $this->classtype;
	}

}

?>